var request = require('request');
var utils = require('./utils.js');
var initUrl = "http://thiner.iteye.com/blog/294736";
var toVisitListA = [initUrl]; // 未访问列表
var toVisitListB = []; // 下一深度

var visitedList = []; // 已访问列表
var maxDepth = 1;
var currDepth = 0;


/**
 * url 标准化
 */
function normalUrl(urlList) {
    // TODO
    return urlList;
}

function filterUnvisited(urlList) {
    var list = [];
    urlList.forEach(function(item) {
        if (visitedList.indexOf(item) < 0) {
            list.push(item);
        }
    });
    return list;
}

var filterList = [];
(function initFilterList() {
    var filterImage = 'jpg,jpeg,gif'; //图片后缀名
    var filterVideo = 'avi,mpeg,mp4,mov,realtime';
    var filterDoc = 'doc,txt,ppt,docx,pptx,xls,xlsx';
    var filterAudio = 'mp3,wmv';
    var filterAll = [filterImage, filterAudio, filterVideo, filterDoc];

    filterAll.forEach(function(oneFilter) {
        var list = oneFilter.split(',');
        filterList = filterList.concat(list);
    });
})();



function filterPostfix(urlList) {
    var list = [];
    urlList.forEach(function(item) {
        if (visitedList.indexOf(item) < 0) {
            var postFix = item.split('.').pop();
            if (filterPrefix.indexOf(postFix) < 0) {
                list.push(item);
            }
        }
    });
    return list;
}

function getAurlToVisit() {
    var url = null;
    if (toVisitListA.length > 0) {
        url = toVisitListA.pop();
        visitedList.push(url);
    } else if (toVisitListB.length > 0) {
        toVisitListA = toVisitListB;
        toVisitListB = [];
        currDepth++;
        console.log('\n\n--------depth up :' + currDepth);
        url = toVisitListA.shift();
    }
    if (currDepth > maxDepth) {
        return null;
    }
    return url;
}

function parseHtmlGetUrl(html, url) {
    var regEx2 = /<a\s+href="([^"]+)"/gi;
    if (!html) return [];
    var urls = [];
    var m;
    while (m = regEx2.exec(html)) {
        if (m && m[1]) {
            var url = utils.getFullUrl(url, m[1]);
            urls.push(url);
        }
    }
    return urls;
}

function getHtmlFromUrl(url) {
    return new Promise(function(fullfil, reject) {
        request(url, function(err, res, body) {
            if (err) fullfil(null);
            else fullfil(body);
        });
    }).then(function(body) {
        return body;
    });
}


function* main() {
    console.log('start');

    while (true) {
        var url = getAurlToVisit();
        console.log('url=' + url);
        if (!url) {
            console.log('to visit list empty');
            break;
        }

        var html = yield getHtmlFromUrl(url);
        console.log(html.length);

        if (html) {
            var urls = parseHtmlGetUrl(html, url);
            // console.log(urls);
            if (!urls || urls.length === 0) {
                continue;
            }
            urls = normalUrl(urls);
            urls = filterUnvisited(urls);
            console.log(urls);
            urls = filterPostfix(urls);
            // console.log(urls);
            toVisitListB = toVisitListB.concat(urls);
        }
    }

    console.log('job done, depth=' + depth + ',visit count=' + visitedList.length);
}

function runGen(gen) {
    var it = gen();
    var ret;

    (function goNext(val) {
        ret = it.next(val);
        if (ret.done) return;

        var v = ret.value;
        if (v instanceof Promise && 'then' in v) {
            v.then(goNext);
        } else if (typeof v === 'function') {
            v(goNext);
        } else {
            process.nextTick(function() {
                setTimeout(goNext.bind(this, v), 5000);
            });
        }
    })();
}


runGen(main);